/*
cd /projects/hsieh_ra/import/
qstata 03_pull_append_wanted_vars.do --memsize=32
**Objective: Pull in all raw ECs for every year and append/save separately */

*Set directories 
local raw_data_dir = "/data/economic/"
local import_dir = "/projects/data_p/import/"
local build_raw_dir = "/projects/data_p/build/raw/"
local ec_dir = "`1'"
/* REDACTED
var2 and var4 below have been renamed from original code files*/
local var2 = "`2'"
local ec_dir2 = "`3'"
local var4 = "`4'"


**Vars are manually inspected and subsetted in Excel.
import excel "`import_dir'EC/`ec_dir'/all_vars_to_keep.xlsx", sheet("Sheet1") firstrow clear
keep if keep == 1
levelsof(name), clean local(sas_vars)

cd "`raw_data_dir'`ec_dir'/"
local filelist: dir . dirs "*"
di `filelist'
foreach p of local filelist {
	if "`p'" != "doc"{
		cd "`p'"
		if "`p'" == "2017" {
			/* REDACTED 
			Code importing the raw file is redacted to avoid disclosing name of raw file.
			Comment discussing structure of raw file is redacted */
			gen YEAR = `p'
			save "`build_raw_dir'`ec_dir2'/individual/`ec_dir'_`p'", replace
		}
		else if "`p'" == "2012"{			
			/* REDACTED 
			Code importing the raw file is redacted to avoid disclosing name of raw file.
			*/
			gen YEAR = `p'
			save "`build_raw_dir'`ec_dir2'/individual/`ec_dir'_`p'", replace
		}
		else {			
			/* REDACTED 
			Code importing the raw file is redacted to avoid disclosing name of raw file.
			*/
			gen YEAR = `p'
			save "`build_raw_dir'`ec_dir2'/individual/`ec_dir'_`p'", replace
		}
	}
	cd "`raw_data_dir'`ec_dir'/"
}

clear
cd "`build_raw_dir'`ec_dir2'/individual/"
local filelist: dir . files "*.dta"
append using `filelist'
save "`build_raw_dir'`ec_dir2'/`ec_dir'_appended.dta", replace

